/**
 * \file webrequest.c
 *       Filename:  webrequest.c
 * =====================================================================================
 * 
 * 
 *    Description:  Implementation of methods for HTTP web access. 
 * 
 *        Version:  1.0
 *        Created:  13/11/2009 00:12:50
 *       Revision:  none
 *       Compiler:  gcc
 * 
 *         Author:  François Hissel (fh), francois.hissel@m4x.org
 *        Company:  
 * 
 * =====================================================================================
 */

#include	<ctype.h>
#include	<stdlib.h>
#include	<stdio.h>
#include	<string.h>
#include	<fcntl.h>
#include	<errno.h>
#include	<netinet/in.h>
#include	<netdb.h>
#include	<sys/socket.h>
#include	<unistd.h>
#include	<sys/types.h>
#include	<sys/stat.h>
#include	<sys/select.h>
#include	<zlib.h>
#include	"webrequest.h"

const char* http_prefix="http://";	//!< Prefix for connection to an HTTP web page
const char* http_headers="User-Agent: Mozilla/5.0 Gecko/2009090216 Ubuntu/9.04 (jaunty) Firefox/3.0.14\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Encoding: gzip,deflate\r\nAccept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n";			//!< Other headers of HTTP request
const char *http_post_form="Content-Type: application/x-www-form-urlencoded\r\n";	//!< Additional headers for POST requests with form body
const char *http_post_multipart="Content-Type: multipart/form-data; boundary=";	//!< Additional headers for POST requests with multipart body

char proxy[STRING_SIZE];	//!< Name of proxy server 
char proxy_port[STRING_SIZE];	//!< Port of proxy server

char* strnstr(char *buffer,size_t num,char *substring,int case_sensitive) {
	size_t len=strlen(substring);
	int index[len];
	int i=1;
	int j=0;
	index[0]=0;
	while (substring[i]!=0) {
		if (j>=0 && (substring[i]==substring[j] || (case_sensitive==0 && toupper(substring[i])==toupper(substring[j])))) index[i++]=++j; else if (j>0) j=index[j]-1; else {index[i++]=j=0;}
	}
	char *res=buffer;
	char *last=buffer+num;
	j=0;
	while (res+len-j<=last) {
		if (*res==substring[j] || (case_sensitive==0 && toupper(*res)==toupper(substring[j]))) {
			++res;++j;
			if (j==len) return res-j;
		} else {
			if (j>0) j=index[j-1]; else ++res;
		}
	}
	return 0;
}

Cookies *get_cookie(Cookies *cl,char *name) {
	while (cl!=0 && strcmp(cl->cookie.name,name)!=0) cl=cl->next;
	return cl;
}

size_t print_cookies(Cookies *c,char *buffer) {
	Cookies *p=c;
	char *b=buffer;
	while (p!=0) {
		char *q=p->cookie.name;
		while (*q!=0) *(b++)=*(q++);
		*(b++)='=';
		q=p->cookie.value;
		while (*q!=0) *(b++)=*(q++);
		p=p->next;
		if (p!=0) {
			(*b++)=';';
			(*b++)=' ';
		}
	}
	return b-buffer;
}

void free_cookies(Cookies *c) {
	Cookies *p=c;
	Cookies *q;
	while (p!=0) {
		q=p->next;
		free(p->cookie.name);
		free(p->cookie.value);
		free(p);
		p=q;
	}
}

/**
 * \brief Write data on a socket with a time-out protection
 *
 * This functions writes data on a socket but adds a timeout so that the operation does not hang and instead return an error code.
 * \param socket Descriptor of the socket
 * \param buf Pointer to the buffer to send on a socket
 * \param len Length of the buffer
 * \param timeout Time-out of the operation, measured in seconds. After this delay, the function returns -2 as an error code
 * \return Number of bytes actually sent on the socket, -1 for a general error (and errno gets the error code) or -2 for a timeout error
 */
ssize_t sock_write(int socket,const void *buf,size_t len,int timeout) {
	fd_set wrfs;
	struct timeval tv;
	FD_ZERO(&wrfs);
	FD_SET(socket,&wrfs);
	tv.tv_sec=timeout;
	tv.tv_usec=0;
	if (select(socket+1,0,&wrfs,0,&tv)!=-1) {
		ssize_t res=write(socket,buf,len);
#ifdef  TRACE
		write(2,buf,res);
#endif
		return res;
	} else return -2;
}

/**
 * \brief Read data from a socket with a time-out protection
 *
 * This function reads data from a socket but adds a timeout so that the operation does not hang and instead return an error code.
 * \param socket Descriptor of the socket
 * \param buf Pointer to the buffer to read from a socket
 * \param len Length of the buffer
 * \param timeout Time-out of the operation, measured in seconds. After this delay, the function returns -2 as an error code
 * \return Number of bytes actually read from the socket, -1 for a general error (and errno gets the error code) or -2 for a timeout error
 */
ssize_t sock_read(int socket,void *buf,size_t len,int timeout) {
	fd_set rdfs;
	struct timeval tv;
	FD_ZERO(&rdfs);
	FD_SET(socket,&rdfs);
	tv.tv_sec=timeout;
	tv.tv_usec=0;
	if (select(socket+1,&rdfs,0,0,&tv)!=-1) {
		ssize_t res=read(socket,buf,len);
		return res;
	} else return -2;
}

int resolve(const char *hostname,const char *port,struct addrinfo **address) {
	struct addrinfo hints;
	memset(&hints,0,sizeof(hints));
	hints.ai_socktype=SOCK_STREAM;
	hints.ai_family=AF_INET;
	return getaddrinfo(hostname,port,&hints,address);
}

/**
 * \brief Send a GET or POST request to a web server
 *
 * This function sends a GET or POST request to a web server, with all the required fields. The program hids itself by publishing that its identity is a Linux Firefox browser (otherwise the web servers may close the connection.) According to the value of the body parameter, the function guesses if it shall send a GET or POST request and adjusts the header. If cl is not NULL, all the cookies in the list pointed are added to the request.
 * \param socket Descriptor of the socket, opened between the local computer and the web server. The connection shall already be opened.
 * \param path Path of the file to access
 * \param host Name of the distant webserver (required if the connection to the server goes through a proxy, but it is always worth specifying, especially for HTTP/1.1 protocols)
 * \param headers Other headers, a constant string in this implementation
 * \param post_headers Special additional headers for a POST request, usually hold the content type. Content length is automatically added by the function and must not belong to this variable
 * \param body Body of the post request. Its size is automatically computed and put in the headers. If body=0, a GET request is sent. If body is not null, a POST request is sent with the body being the string pointed by this variable.
 * \param cl Pointer to the list of cookies
 * \return 0 if the function was successful, another value otherwise
 */
int send_request(int socket,const char *path,const char *host,const char *headers,const char *post_headers,const char *body,Cookies *cl) {
	ssize_t bsize;	// Length of the header
	size_t size=0;	// Total length of body
	if (body!=0) size=strlen(body);
#ifdef TRACE
	fprintf(stderr,"\n*************\n");
#endif
	unsigned char cookies[BUFFER_SIZE];	// List of cookies in a format compatible with the HTTP protocol
	size_t csize=0;	// Size of the list of cookies
	if (cl!=0) csize=print_cookies(cl,(char*)cookies);
	cookies[csize]=0;
	char clength[20];	// Length of the body, printed in a string
	sprintf(clength,"%zu",size);
	if (body==0) {
		if (cl==0) bsize=26+strlen(path)+strlen(host)+strlen(headers); else bsize=36+strlen(path)+strlen(host)+strlen(headers)+csize;
	} else {
		if (cl==0) bsize=56+1*strlen(path)+2*strlen(host)+strlen(headers)+strlen(http_prefix)+strlen(clength)+strlen(post_headers); else bsize=66+1*strlen(path)+2*strlen(host)+strlen(headers)+strlen(http_prefix)+strlen(clength)+strlen(post_headers)+csize;
	}
	unsigned char buffer[bsize];
	if (body==0) {
		if (cl==0) sprintf((char*)(buffer),"GET %s HTTP/1.1\r\nHost: %s\r\n%s\r\n",path,host,headers);
		else sprintf((char*)(buffer),"GET %s HTTP/1.1\r\nHost: %s\r\n%sCookie: %s\r\n\r\n",path,host,headers,cookies);
	}
	else {
		if (cl==0) sprintf((char*)(buffer),"POST %s HTTP/1.1\r\nHost: %s\r\n%sReferer: %s%s\r\n%sContent-Length: %s\r\n\r\n",path,host,headers,http_prefix,host,post_headers,clength);
		else sprintf((char*)(buffer),"POST %s HTTP/1.1\r\nHost: %s\r\n%sReferer: %s%s\r\n%sContent-Length: %s\r\nCookie: %s\r\n\r\n",path,host,headers,http_prefix,host,post_headers,clength,cookies);
	}
	if (sock_write(socket,buffer,bsize-1,TIMEOUT)<bsize-1) return -1;
	if (body!=0) {
		size_t written=0;	// Number of bytes of the body really written on the socket
		const char *p=body;
		while (written<size) {
			if (size-written>=BUFFER_SIZE) bsize=sock_write(socket,p,BUFFER_SIZE,TIMEOUT); else bsize=sock_write(socket,p,size-written,TIMEOUT);
			if (bsize<0) return -1;
			written+=bsize;
			p+=bsize;
		}
	}
	return 0;
}

/**
 * \brief Get a standardized path name to be used in a Web HTTP request
 *
 * This function checks if a proxy has been configured. If it is the case, the fields of the GET request have to be adjusted. And so it calculates the new value that should be passed to the GET or POST address.
 * \param host Host name
 * \param path Path of the requested address on the web server
 * \return Path to be used in the request. New memory is allocated by this function and should be released when it is not needed any longer
 */
char *get_http_path(const char *host,const char *path) {
	char *ppath;
	int dec;
	if (proxy[0]==0) {	// If a proxy is selected, use it in the request, else connect directly to the website
		if (strstr(path,"://")!=0) dec=strlen(http_prefix)+strlen(host); else dec=0;
		ppath=(char*)malloc(strlen(path)-dec+1);
		strcpy(ppath,path+dec);
	} else {
		if (strstr(path,"://")!=0) {
			ppath=(char*)malloc(strlen(path)+1);
			strcpy(ppath,path);
		} else {
			ppath=(char*)malloc(strlen(http_prefix)+strlen(host)+strlen(path)+1);
			strcpy(ppath,http_prefix);
			strcat(ppath,host);
			strcat(ppath,path);
		}
	}
	return ppath;
}

int upload_request(struct addrinfo *hostinfo,const char *host,const char *path,const char *filename,FILE *file,ssize_t filesize,Cookies *cl) {
#ifdef TRACE
	fprintf(stderr,"\n*************\n");
#endif	
	char *ppath=get_http_path(host,path);
	int socket_fd;
	socket_fd=socket(AF_INET,SOCK_STREAM,0);
	if (connect(socket_fd,hostinfo->ai_addr,hostinfo->ai_addrlen)==-1) {	// Connect to server
		fprintf(stderr,"Can't connect to server : %s\n",hostinfo->ai_canonname);
		free(ppath);
		return -1;
	}
	char *sep="----------------------------6232c358f158";	// Separator between parts of the body content
	size_t sepsize=strlen(sep);
	ssize_t size=filesize+strlen(filename)+98+2*sepsize;	// Total length of body
	unsigned char cookies[BUFFER_SIZE];	// List of cookies in a format compatible to the HTTP protocol
	size_t csize=0;	// Size of the list of cookies
	if (cl!=0) csize=print_cookies(cl,(char*)cookies);
	cookies[csize]=0;
	char clength[20];	// Length of the body, printed in a string
	sprintf(clength,"%zu",size);
	ssize_t bsize=91+strlen(ppath)+1*strlen(host)+strlen(http_headers)+sepsize+strlen(clength);	// Length of the header
	if (cl!=0) bsize+=10+csize;
	unsigned char buffer[bsize];
	size_t s=sprintf((char*)buffer,"POST %s HTTP/1.1\r\n%sHost: %s\r\n",ppath,http_headers,host);
	free(ppath);
	if (cl!=0) s+=sprintf((char*)buffer+s,"Cookie: %s\r\n",cookies);
	s+=sprintf((char*)buffer+s,"Content-Type: multipart/form-data; boundary=%s\r\nContent-Length: %s\r\n\r\n",sep,clength);
	if (sock_write(socket_fd,buffer,bsize-1,TIMEOUT)<bsize-1) return -1;
	unsigned char body[BUFFER_SIZE];
	s=sprintf((char*)body,"--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\nContent-Type: text/plain\r\n\r\n",sep,filename);
	bsize=sock_write(socket_fd,body,s,TIMEOUT);
	if (bsize<0) return -1;
	s=0;
	unsigned char *p;
	while (s<filesize) {
		if (filesize-s>=BUFFER_SIZE) bsize=fread(body,1,BUFFER_SIZE,file); else bsize=fread(body,1,BUFFER_SIZE,file);
		size_t written=0;
		p=body;
		while (written<bsize) {
			csize=sock_write(socket_fd,p,bsize-written,TIMEOUT);
			if (csize<0) return -1;
			written+=csize;
			p+=csize;
		}
		s+=bsize;
	}
	s=sprintf((char*)body,"\r\n--%s--\r\n",sep);
	bsize=sock_write(socket_fd,body,s,TIMEOUT);
	if (bsize<0) return -1;
	return 0;
}

int send_custom(int *socket_fd,struct addrinfo *hostinfo,const char *host,const char *path,const char *headers,const char *body,Cookies *cl) {
	// Connection to the server
	char *ppath=get_http_path(host,path);
	int sock;
	sock=socket(AF_INET,SOCK_STREAM,0);
	if (connect(sock,hostinfo->ai_addr,hostinfo->ai_addrlen)==-1) {	// Connect to server
		fprintf(stderr,"Can't connect to server : %s\n",hostinfo->ai_canonname);
		free(ppath);
		return -1;
	}
	// Sending of the request
	int code;
	if (send_request(sock,ppath,host,http_headers,headers,body,cl)!=0) {
		fprintf(stderr,"Couldn't send request to server : %s, error code %i.\n",host,errno);
		code=-1;
	} else code=0;
	free(ppath);
	if (socket_fd!=0) *socket_fd=sock; else close(sock);
	return code;
}

long request_addr(FILE* fd,struct addrinfo *hostinfo,const char *host,const char *path,const char *body,Cookies **cl) {
	if (fd==0) return 0;	// Do not try to download the server response if the file pointer does not hold any file
	// Connect to the server and send the request
	int socket_fd;
	int code=send_custom(&socket_fd,hostinfo,host,path,http_post_form,body,*cl);
	if (code!=0) {
		close(socket_fd);
		return code;
	}
	// Read response from socket and uncompress it if necessary
#ifdef TRACE
	fprintf(stderr,"\n-----------------\n");
#endif	
	unsigned char buffer[BUFFER_SIZE];
	unsigned long bsize;	// Size of body already read
	//unsigned long size=0;
	long readsize=0;	// Number of bytes read from site. At the end of the procedure, will be compared to the expected number which is given by datasize.
	char start=1;	// Boolean value to check if we are receiving the first block of data. If this is the case, we should read the header to find some description of the following content
	unsigned long int datasize=0;	// Expected number of bytes to be read for the content
	long xsize=0;	// Size of the file after inflation of compressed stream
	int ret;	// Return value of zlib procedures
	unsigned have;	// Amount of data feeded by inflate
	z_stream strm;	// Internal structure describing state of uncompression algorithm
	char encoded=0;	// Tells whether the stream is encoded in gzip format
	unsigned char *pos;	// Start of buffer to write
	char *p,*q,*r;
	int wsize;	// Number of bytes to write
	int error=0;
	unsigned char out_buffer[BUFFER_SIZE];
	do {
		//readsize=0;
		bsize=sock_read(socket_fd,buffer,BUFFER_SIZE,TIMEOUT);	// Read a block of data
		if (bsize<0) {
			fprintf(stderr,"Error while reading from server %s, error code %i.\n",host,errno);
			close(socket_fd);
			return -1;
		} else if (bsize>0) {
			if (start && strncmp((char*)buffer,"HTTP",4)==0) {	// If it is the first block and if a header is detected, reads the header to find some description of the content (encoding and size)
				start=0;
				// Get the HTTP return code
				pos=buffer;
				while (*pos!=' ') ++pos;
				++pos;
				int http_return=atoi((char*)pos);
				// Read headers
				pos=buffer;
				while (*pos!='\r' && *pos!='\n') {
					p=strchr((char*)pos,':');
					q=strchr((char*)pos,'\n')+1;	// Go to next line
					if (p!=0 && p<q) {	// We are reading a header line
						if (strncmp((char*)pos,"Content-Length",14)==0) datasize=atoi((char*)pos+16);	// Read length of body
						else if (strncmp((char*)pos,"Content-Encoding",16)==0) {	// Find if content is encoded using gzip
							++p;
							while (isspace(*p)) ++p;
							if (strncmp((char*)p,"gzip",4)==0) {
								encoded=1;
								strm.zalloc=Z_NULL;
								strm.zfree=Z_NULL;
								strm.opaque=Z_NULL;
								strm.avail_in=0;
								strm.next_in=Z_NULL;
								ret=inflateInit2(&strm,0x1f);
							}
						} else if (cl!=0 && strncmp((char*)pos,"Set-Cookie",10)==0) {
							++p;
							while (isspace(*p)) ++p;
							r=strchr((char*)p,'=');
							char *cname=malloc(r-p+1);
							strncpy(cname,p,r-p);cname[r-p]=0;
							p=r+1;
							while (*p!='\r' && *p!=';' && *p!='\n') p++;
							Cookies *cook=get_cookie(*cl,cname);
							if (cook==0) {
								cook=malloc(sizeof(Cookies));
								cook->cookie.name=cname;
								cook->cookie.value=malloc(p-r);
								strncpy(cook->cookie.value,r+1,p-r-1);cook->cookie.value[p-r-1]=0;
								cook->next=*cl;
								*cl=cook;
							} else {
								free(cook->cookie.name);
								cook->cookie.name=cname;
								free(cook->cookie.value);
								cook->cookie.value=malloc(p-r);
								strncpy(cook->cookie.value,r+1,p-r-1);cook->cookie.value[p-r-1]=0;
							}
						}
					}
					pos=(unsigned char*)q;
				}
				pos+=2;
				wsize=bsize-(pos-buffer);
#ifdef TRACE
				write(2,buffer,pos-buffer);
#endif
				if (http_return!=200) return -http_return;	// If the file is found on the server, the HTTP return code should be 200. Otherwise, exit function and return the opposite of the HTTP return code
			} else {
				pos=buffer;
				wsize=bsize;
			}
			readsize+=wsize;
			// Write data to file, uncompressing it if it is encoded
			if (encoded) {
				strm.avail_in=wsize;
				if (strm.avail_in==0) break;
				strm.next_in=pos;
				do {
					strm.avail_out=BUFFER_SIZE;
					strm.next_out=out_buffer;
					ret=inflate(&strm,Z_NO_FLUSH);
					have=BUFFER_SIZE-strm.avail_out;
					size_t wr=fwrite(out_buffer,1,have,fd);
#ifdef TRACE
					write(2,out_buffer,have);
#endif
					if (wr==-1 || ret==Z_DATA_ERROR || ret==Z_MEM_ERROR) {
						inflateEnd(&strm);
						error=-1;
					} else xsize+=wr;
				} while (error==0 && strm.avail_out==0);
			} else {
				error=fwrite(pos,1,wsize,fd);
#ifdef TRACE
				write(2,pos,wsize);
#endif
			}
			if (error==-1) {
				fprintf(stderr,"Error while writing file, error code %i.\n",errno);
				close(socket_fd);
				return -1;
			} else {
				//size+=readsize;
			}
		}
	} while (readsize<datasize && bsize!=0);
	// Close open descriptors and exit
	if (encoded==1) inflateEnd(&strm);
	close(socket_fd);
	//fsync(fd);
	fflush(fd);
	if (datasize>0 && datasize!=readsize) {	// This may happen only if the headers of the page displayed a size different from the actual size read from the socket. The file is however saved on the disk, in order to allow manual fixing.
		fprintf(stderr,"Connection problem : file could not be fully downloaded. Expected size : %li bytes, actual size : %li bytes.\n",datasize,readsize);
		return -1;
	}
	return xsize;
}

long request(FILE* fd,const char *host,const char *port,const char *path,const char *body,Cookies **cl) {
	const char *phost;
	const char *pport;
	char *ppath;
	if (proxy[0]==0) {	// If a proxy is selected, use it in the request, else connect directly to the website
		phost=host;
		pport=port;
		ppath=(char*)malloc(strlen(path)+1);
		strcpy(ppath,path);
	} else {
		phost=proxy;
		pport=proxy_port;
		ppath=(char*)malloc(strlen(http_prefix)+strlen(host)+strlen(path)+1);
		strcpy(ppath,http_prefix);
		strcat(ppath,host);
		strcat(ppath,path);
	}
	struct addrinfo *hostinfo;	// Resolve address of proxy or website
	if (resolve(phost,pport,&hostinfo)!=0) {
		fprintf(stderr,"Can't resolve address : %s\n",phost);
		free(ppath);
		return -1;
	}
	int code=request_addr(fd,hostinfo,host,path,body,cl);
	freeaddrinfo(hostinfo);
	return code;
}
